In [1]:
import pandas as pd
import numpy as np
import seaborn as sns
from scipy import stats
import sklearn as sk
import sklearn.naive_bayes as sknb
import sklearn.metrics as skm
import sklearn.cluster as skc
import sklearn.decomposition as skd
import sklearn.preprocessing as skp
import statsmodels.formula.api as smf
import statsmodels.api as sm
import sklearn.tree as sktree
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
sns.set_context("notebook") # make figures fit
from pylab import rcParams
import matplotlib.pyplot as plt
from statsmodels.graphics.mosaicplot import mosaic
from IPython.display import Image
from IPython.core.display import HTML
import sklearn.externals.six as sksix
import IPython.display as ipd
import os
import pydot_ng as pydot
# load the bokeh stuff
from bokeh.io import output_notebook, show
from bokeh.plotting import figure
output_notebook()
from bokeh.layouts import column, row
from bokeh.charts import Scatter
from bokeh.charts import TimeSeries
from bokeh.charts import Bar
from bokeh.charts import Histogram
from bokeh.charts import BoxPlot
from bokeh.layouts import gridplot
from bokeh.models import HoverTool
from bokeh.models import ColumnDataSource
exec(open('useful.py').read())
# make the Pandas tables a little more readable
from IPython.core.display import HTML
css = open('style-table.css').read() + open('style-notebook.css').read()
HTML('<style>{}</style>'.format(css))
Out[1]:
LOAD IN THE CSVS YOU WILL NEED¶
In [3]:
Master = pd.read_csv("Master.csv")
Salaries = pd.read_csv("Salaries.csv")
Teams = pd.read_csv("Teams.csv")
Batting = pd.read_csv("Batting.csv", parse_dates = ['yearID'])
Appearances = pd.read_csv("Appearances.csv", parse_dates=['yearID'])
CREATE COPIES OF TABLES TO CREATE THE TABLE YOU NEED¶
In [4]:
#MasterSel will comprise of only the variables you are interested in
# I am particularly interested in weight/ might take birth into account
MasterSel = Master[['playerID','weight','height','birthCountry']]
In [5]:
# quickly examine Master Sel
MasterSel.head(2)
Out[5]:
In [6]:
# looked at NA's there are only a few, so I feel comfortable excluding them
# from my analysis
MasterSel = MasterSel.dropna()
In [7]:
# Similiar to MasterSel I will narrow down the variables I want from
# the batting table
# BattingSel is also a cut down of the Batting csv
BattingSel = Batting[['playerID','yearID','teamID','G','AB','R','H','RBI','HR','2B','3B','SO']]
In [8]:
# for further analysis we will be checking
# batting statistics, if we don't have batting data for them we will drop them
# when merging columns will make sure to merge on only those players in batting
BattingSel = BattingSel.dropna()
In [9]:
# batting table has the items you want now
BattingSel.head(2)
Out[9]:
In [10]:
# Positions is the appearances table cut down
# the variables in this dataframe are the number of times
# a player has played such position in that given year
# this will later give us the position of each player
Positions = Appearances[['yearID','playerID', 'G_all','G_p', 'G_c', 'G_1b', 'G_2b', 'G_3b', 'G_ss', 'G_of']]
In [11]:
Positions.head(2)
Out[11]:
In [12]:
#set all indexes to "playerid" so you can later merge them
MasterSel = MasterSel.set_index('playerID')
BattingSel = BattingSel.set_index('playerID')
Positions= Positions.set_index('playerID')
In [13]:
# MasterPos will now have POSITIONAL data and weight,height, and birth country
MasterPos = pd.merge(MasterSel, Positions, left_index=True, right_index=True, how='inner')
In [14]:
#MasterPos.describe()
MasterPos.head()
Out[14]:
In [15]:
# Creating a column to store the max position of games played
# then using pmax to create actual labels!
MasterPos['pmax'] = MasterPos[['G_p', 'G_c', 'G_1b', 'G_2b', 'G_3b', 'G_ss','G_of']].max(axis=1)
# Creating a column called position to store labels for classification
MasterPos['position']= ''
In [16]:
# creating LABELS for positional data
counter = 0
for row in MasterPos.iterrows():
if row[1]['pmax'] == row[1]['G_p']:
MasterPos.ix[counter,'position'] = 'Pitcher'
elif row[1]['pmax'] == row[1]['G_c']:
MasterPos.ix[counter,'position'] = 'Catcher'
elif row[1]['pmax'] == row[1]['G_1b']:
MasterPos.ix[counter,'position'] = 'FirstBase'
elif row[1]['pmax'] == row[1]['G_2b']:
MasterPos.ix[counter,'position'] = 'SecondBase'
elif row[1]['pmax'] == row[1]['G_3b']:
MasterPos.ix[counter,'position'] = 'ThirdBase'
elif row[1]['pmax'] == row[1]['G_ss']:
MasterPos.ix[counter,'position'] = 'ShortStop'
elif row[1]['pmax'] == row[1]['G_of']:
MasterPos.ix[counter,'position'] = 'Outfielder'
counter += 1
In [17]:
# MasterPos now has positional labels
# filter our the uneccessary columns that helped
# determine position for that year
MasterPos = MasterPos[['yearID','weight','height','birthCountry','G_all','position']]
In [18]:
# MasterPos.position.value_counts() examined positions
In [19]:
MasterPos.head(2) # now that masterpos has the information you would like
Out[19]:
In [20]:
Position_df = MasterPos[['yearID','position']]
In [21]:
# you want the weight of each player
Weight_df = MasterPos[['weight','height','birthCountry']]
Weight_df = Weight_df.drop_duplicates()
Question 1: What is the distribution of weight among baseball players ? More specifically, their BMI (Body Mass Index)¶
In [22]:
weighthist = Histogram(MasterPos, 'weight', bins = 30, xlabel='Weight', ylabel='# of Players', plot_width =600, title='Weight Distribution',color = 'navy')
show(weighthist)
from the histogram above it appears that the distribution of weight appears to be normally distributed, but to make sure we will examine the fourplot of weight¶
In [23]:
# the histogram looks the same before
# more importantly, we are examining the QQplot here to better visualize normality
fourPlot(MasterPos['weight'])
the QQ plot confirms our histogram and we see here that weight is normal¶
Because we are also taking height into account for BMI we must now examine height :¶
once I plotted the height's histogram and looked at the height's qqplot I confirmed that height was also very normal
In [24]:
heighthist = Histogram(MasterPos, 'height', bins = 20, xlabel='height', ylabel='# of Players', plot_width =600, title='Height Distribution',color = 'red')
show(heighthist)
In [25]:
fourPlot(MasterPos['height']) # height is also very normal! QQ plot looks good
In [26]:
## description of BMI
In [27]:
# what does their BMI look like
MasterPos['BMI'] = (MasterPos['weight']/np.power(MasterPos['height'],2)) *703
BMIHist = Histogram(MasterPos,'BMI',bins=30, ylabel='# of Players', title = 'BMI Distribution',color='purple')
show(BMIHist)
In [28]:
# anything below 18.5 is underweight, 25 overweight, and anything past 30 is obsese
perofBMI = [100 - stats.percentileofscore(MasterPos['BMI'],i) for i in [18.5,26,30,40]]
perofBMI
Out[28]:
In [29]:
100 - perofBMI[0] # 0.025 are underweight
# 27% are overweight and 2.49 are considered obese
Out[29]:
it appears that .025% of players are underweight, 27% are overweight, and 2.49% are considered obese¶
Question #2: Have baseball players been getting larger over time?¶
lets look at the average weight of baseball players every year because the data stems from 1871 - 2016 this will easier to plot and easier to interpret
In [31]:
# lets look at the average weight of baseball players every year
AvgLBTS = MasterPos.groupby('yearID')[['weight']].mean()
In [32]:
AvgLBTS.head(2)
Out[32]:
In [33]:
# not very pretty, but this is the timeseries of average weight over time
ts = AvgLBTS["weight"]
plt.plot(ts)
Out[33]:
ROLLING AVERAGE FOR MY TIMESERIES ON AVERAGE WEIGHT OVER TIME - theres a clear trend but rolling average confirms this too.¶
In [34]:
r = AvgLBTS.rolling(window=10) # windows is you days you want of rolling ; the bigger the window the smaller the varability
AvgLBTS.weight.plot(color='gray')
r.mean().weight.plot(color='red')
Out[34]:
different ways to visualize my timeseries¶
In [35]:
AvgLBTSN_IX = AvgLBTS.reset_index()
AvgLBTSN_IX.head(2)
Out[35]:
In [36]:
TS = TimeSeries(AvgLBTSN_IX,
x= "yearID", y='weight', title="Timeseries", ylabel='Average Weight',xlabel='Year',plot_width=400,color='LightBlue', legend=False)
show(TS)
In [37]:
AvgLBTSN_IX_2 = AvgLBTSN_IX.copy()
In [38]:
AvgLBTSN_IX_2['yearID'] = AvgLBTSN_IX_2['yearID'].dt.strftime('%Y')
In [39]:
#specify data source
source = ColumnDataSource(AvgLBTSN_IX_2)
# what pops up to hover
tooltips = [('weight', '@weight'),
('year', '@yearID')]
hover = HoverTool(tooltips=tooltips)
p = figure(plot_width=700,plot_height=450,title="MLB weight over the years", x_axis_label='Year',y_axis_label = "Average Weight (in Pounds)")
# Add the hover tool
p.add_tools(hover)
# # Populate glyphs
p.circle(x='yearID', y='weight', size=7, alpha=0.5, source=source)
show(p)
Question 3: Does this increase in trend look different if we look at the average weight yearly by position?¶
In [40]:
AvgLBPos = MasterPos.groupby(['yearID','position'])[['weight']].mean()
AvgLBPos = AvgLBPos.unstack().stack().T.stack()
In [41]:
PLOT = AvgLBPos.T.plot(title='Average Yearly Weight by Position')
PLOT.set_xlabel('Year')
PLOT.set_ylabel('Average Weight')
PLOT.legend(loc = 'upper left',fontsize = 7)
Out[41]:
interestingly we can see where different trend lines are. First base is really high above, first base man usually don't cover much ground they stay near first base they dont need to be light , shortstop trend is lower among positions they typically cover more ground and thus being light would be advantageous¶
if i had more time I would remove the trend and preform a cross correlation among my position time series to see if they are all correlated or if there is a difference among them. Would specualte that they would be correlated, but further analysis would be required¶
Question 4: after exaiming weight and position, can we classify baseball players position by their weight?¶
Naive Bayes¶
In [43]:
# train the model
gnb_model = sknb.GaussianNB()
# given weight, predict position
gnb_model.fit(MasterPos['weight'].reshape(len(MasterPos['weight']),1),MasterPos["position"])
y_pred = gnb_model.predict(MasterPos['weight'].reshape(len(MasterPos['weight']),1)) # fix this
MasterPos['predicted_nb'] = y_pred
In [44]:
### skm.accuracy_score(...)
skm.accuracy_score(y_true=MasterPos["position"],y_pred=MasterPos["predicted_nb"])
Out[44]:
Performed a navie bayes with weight and position. Unfortunately, my accuracy score was .44, which is a bad accuracy score.. it's very low. Moving forward by preforming a decision tree classifer¶
bad accuracy score, try decision tree classifier¶
Try Decision Tree¶
again here we are looking at classify position by weight¶
In [46]:
### Weight alone
dt_model = sktree.DecisionTreeClassifier(max_depth = 2, criterion='entropy')
dt_model.fit(MasterPos[['weight']],MasterPos['position'])
predicted_labels = dt_model.predict(MasterPos[['weight']])
MasterPos['predicted_label_tree'] = predicted_labels
In [47]:
### Weight alone
dot_data = sksix.StringIO()
sktree.export_graphviz(dt_model, out_file=dot_data,
feature_names=['weight'],
filled=True)
graph = pydot.graph_from_dot_data(dot_data.getvalue())
ipd.Image(graph.create_png())
Out[47]:
In [48]:
skm.accuracy_score(y_true=MasterPos["position"],y_pred=MasterPos['predicted_label_tree'])
Out[48]:
In [49]:
cfmat = skm.confusion_matrix(
y_true = MasterPos["position"],
y_pred = MasterPos['predicted_label_tree']
)
sns.heatmap(cfmat, annot=True)
Out[49]:
From the looks of my decision tree and my accuracy score, the model appears to be a bad method to predict position. Accuracy score is .44 again. Examining my confusion matrix it appears that my decision tree is guessing and when it's guessing ( i checked my predicted tree labels) it's guessing all pitchers¶
Moved forward by looking at weight and height as features to predict position however, I got the same results as with weight alone... we see here the bad accuracy score of .44 again and a confusion matrix once again that shows guessing. Moving forward by looking at a distribution of weight by positions.¶
In [50]:
### Perhaps Height and Weight ?
dt_model = sktree.DecisionTreeClassifier(max_depth = 2, criterion='entropy')
dt_model.fit(MasterPos[['weight','height']],MasterPos['position'])
predicted_labels = dt_model.predict(MasterPos[['weight','height']])
MasterPos['predicted_label_tree'] = predicted_labels
In [51]:
dot_data = sksix.StringIO()
sktree.export_graphviz(dt_model, out_file=dot_data,
feature_names=['weight','height'],
filled=True)
graph = pydot.graph_from_dot_data(dot_data.getvalue())
ipd.Image(graph.create_png())
Out[51]:
In [52]:
skm.accuracy_score(y_true=MasterPos["position"],y_pred=MasterPos['predicted_label_tree'])
Out[52]:
In [53]:
cfmat = skm.confusion_matrix(
y_true = MasterPos["position"],
y_pred = MasterPos['predicted_label_tree']
)
sns.heatmap(cfmat, annot=True)
Out[53]:
same thing is occuring when I look at weight and height as features to predict position¶
In [54]:
MasterPos.groupby('predicted_label_tree').sum()
Out[54]:
most labels were being predicted as pitcher¶
looking at the labels and the confusion matrix we can see that the classifier is guessing
Why didn't my decision tree work?¶
In [57]:
PositionDist = MasterPos.copy()
In [58]:
PositionDist.sort_values(by='position', inplace=True)
hist = Histogram(PositionDist, values='weight', color='position',
title="Weight Distribution by Position", legend='top_right')
show(hist)
the distribution of weights are all heavily overlapping, thus our decision tree classifier was having a hard time creating meaningful cuts in the data. Also, the count of pitchers in this data set appears to be very high ( makes sense a lot of teams usually scout for pitchers) but this may also be why our decision tree predicts pitcher so often.¶
Question #5 ( CORRELATION MATRIX) Is there a correlation between a player’s weight and his batting statistics. Does size matter in terms of batting?¶
In [59]:
# create the table you want batting stats and weight
BatWeight = pd.merge(BattingSel, Weight_df, left_index=True, right_index=True, how='inner')
In [60]:
BatWeight.head(2)
Out[60]:
In [61]:
# want players who have had the opportunity to bat @ least 100 times
# even if they got 0's but at least they had the opportunity to preform.
BatWeightH = BatWeight[BatWeight.AB > 100]
In [62]:
BatWeightH.head(2)
Out[62]:
In [63]:
batstat = ['AB','R','H','HR','RBI','3B','2B','SO','weight','height']
cormat = BatWeightH[batstat].corr(method='spearman')
cormat
Out[63]:
In [68]:
# HR's seems to have the strongest correlation with weight .47
sns.heatmap(cormat,linewidths=.5,annot=True)
sns.set(font_scale=.5)
here we note that weight is correlated .47 with homeruns this an interesting finding! weight is also correlated with strikeouts .38 and weight is correlated with 3B hits -.25.¶
Perhaps when you weight more you have more power meaning you hit farther, but you don't necessary swing faster ( hard to pull a 3 base hit) or maybe you don't run fast enough to get 3rd base hits?
Question 6 : Again, what if we dig deeper and look at batting statistics in term of positions: What do batting statistics look like for players who play first base? What about players who play shortstop? (continue if you find more interesting things). Looking for the relationship between batting variables in given subsets of position dataframes.¶
still want players who have batted at least 100 times, the reason being you want to look at batting statistics for players who have at least had the opportunity to bat (even if they suck)¶
In [69]:
BatWeightH_2 = BatWeightH.copy()
BatWeightH_2 = BatWeightH_2.reset_index() # do this to be able to merge
In [70]:
Position_df = Position_df.reset_index() # have to do this to merge
In [71]:
Pos_Bats = pd.merge(BatWeightH_2,Position_df,on=['playerID','yearID'])
Pos_Bats = Pos_Bats.set_index('playerID')
In [72]:
Pos_Bats.head(3)
Out[72]:
subset of dataframe by positions ( looking at firstbase and shortstop first )¶
In [73]:
FB = Pos_Bats[Pos_Bats['position'] == "FirstBase"]
SS = Pos_Bats[Pos_Bats['position'] == "ShortStop"]
In [74]:
#let's look at 5 main batting stats & their weight
Big5_W = ['R','HR','RBI','H','SO','weight']
In [75]:
# firstbase
sns.pairplot(FB[Big5_W])
Out[75]:
variables are very skewed, but we do see there is a correlation between RBI's and hits, Strike outs and Homeruns, Weight and strike outs. May be heavier players take longer to swing?¶
correlation matrix below confirms what we saw in the splom. only interesting things with weight here is strike outs and homeruns which we saw when we were looking at all players. This isn't very interesting¶
In [76]:
# CONFIRMING AND BETTER VISUALIZING THE CORRELATIONS WE SAW IN SPLOM
cormat = FB[Big5_W].corr(method='spearman') # spearman because vars are skewed
sns.heatmap(cormat,linewidths=.5,annot=True)
sns.set(font_scale=.4)
In [77]:
# shortstop
sns.pairplot(SS[Big5_W])
# again variables look skewed
Out[77]:
strong correlation between hits and runs, so and weight, rbis and hits, weight and homeruns¶
once again, this finding is similar to the finding we saw in all players. The correlation matrix shows the same. Not interesting¶
In [78]:
# CONFIRMING AND BETTER VISUALIZING THE CORRELATIONS WE SAW IN SPLOM
cormat = SS[Big5_W].corr(method='spearman')
sns.heatmap(cormat,linewidths=.5,annot=True)
sns.set(font_scale=.4)
Question 7: In Question 5 we saw the strongest correlation to weight was HomeRuns, can we use homeruns as a dependent variable in linear regression: AVGHR ~ Weight¶
In [79]:
AVGHR = BatWeightH.groupby(BatWeightH.index)['HR'].mean()
In [80]:
BatWeightH.HR.hist()
Out[80]:
In [81]:
AVGHR.hist()
Out[81]:
In [82]:
# add 1 so we dont take the log 0
qntls, xr = stats.probplot(np.log(BatWeightH.HR+1), fit=False)
sns.regplot(xr,qntls)
Out[82]:
In [85]:
qntls, xr = stats.probplot(AVGHR+1, fit=False)
sns.regplot(xr,qntls)
Out[85]:
we cant use home runs because as we can see from the analyses above homeruns is not normal and log transforming the data doesn't correct the distribution. I didn't want to filter out 0's because as I previously mentioned I looked at players who had the opportunity to bat at least 100 times and thus a 0 for a home run for them doesn't mean they didn't get the chance to hit a homerun, but they didn't do well in terms of homeruns then.¶
Question 8: What does the trend for average home runs look like?¶
In [86]:
AvgHRByYr = BatWeight.groupby('yearID')[['HR']].mean()
AvgHRByYr = AvgHRByYr.reset_index()
AvgHRByYr.head(2)
Out[86]:
In [87]:
from bokeh.layouts import row
TS = TimeSeries(AvgLBTSN_IX,
x= "yearID", y='weight', title="Timeseries", ylabel='Average Weight',xlabel='Year',plot_width=400,color='LightBlue', legend=False)
TS2 = TimeSeries(AvgHRByYr,
x= "yearID", y='HR', title="Timeseries", ylabel='Average HomeRuns',xlabel='Year',plot_width=400,color='LightCoral', legend=False)
show(row(TS,TS2))
wanted to look at them side by side but home runs looks a bit odd here¶
there seems to be alot of seasonality and trend here in homeruns, if time allows go back and correct for it.¶
In [88]:
AvgHRByYr['yearID'] = AvgHRByYr['yearID'].dt.strftime('%Y')
In [89]:
source1 = ColumnDataSource(AvgHRByYr)
tooltips1 = [('homeruns', '@HR'),
('year', '@yearID')]
hover1 = HoverTool(tooltips=tooltips1)
p1 = figure(plot_width=700,plot_height=450,title="MLB homeruns over the years", x_axis_label='Year',y_axis_label = "Average Homeruns")
# Add the hover tool
p1.add_tools(hover1)
# # Populate glyphs
p1.circle(x='yearID', y='HR', size=7, alpha=0.5, color='LightCoral', source=source1)
show(p1)
Question 9: Look at Salary data? Has Salary been increasing over the years¶
In [90]:
Salaries.describe()
Out[90]:
In [91]:
Salaries.dtypes
Out[91]:
In [92]:
Salaries.head(3)
Out[92]:
In [99]:
np.log(Salaries[['salary']] + 1).hist()
Out[99]:
In [100]:
logSal = np.log(Salaries[['salary']] + 1)
In [101]:
fourPlot(logSal)
unfortuntely, salaries are highly skewed and not normal looking at the histogram and QQplot also would have to normalize by inflation if I would like to make better comparisons.¶
In [102]:
s = BoxPlot(Salaries, values='salary', label='yearID', color='yearID',
title="Salary Box Plot by Year", width = 900, height=900)
show(s)
can see how very skewed baseball data salary can be! Think about interpreting it via a rank or looking at 1 year¶
In [103]:
# Salaries.groupby('yearID').mean()
# # salaries started at 476,299 - 4,396,409
# what about their median
# Salaries.groupby('yearID').median()
# the median is 400,000 - 1,500,000
for a later question, found that CUBS TEAM ID changed from CHN to CHC¶
In [176]:
Salaries['teamID'] = Salaries['teamID'].replace('CHC','CHN')
Question 10: Look at Team data: winning percentage??? Can we predict team wins from their run differental?¶
In [114]:
Teams.head(3)
Out[114]:
In [116]:
Teams.columns
Out[116]:
correlation matrix? / fix teams
In [104]:
cormat = Teams[['Rank','W','L','R','RA','G','H','HR']].corr(method='spearman')
cormat
Out[104]:
strongest correlation with wins was run and hits, look into run differential¶
In [105]:
Teams['rundiffer'] = (Teams['R'] - Teams['RA']) / Teams.G
teams.G is the number of games, so I divided by this to normalize for seasons where there may have been less games¶
In [106]:
Teams['WinPCT'] = Teams['W'] / Teams['L']
In [107]:
Teams20 = Teams[Teams['yearID'] > 1999]
model1 is looking at 2000 through 2016¶
In [108]:
model1 = smf.ols('WinPCT ~ rundiffer', data=Teams20).fit()
model1.summary()
Out[108]:
the model accounts for .84 percent of the variation in win percentage. My predictor run differential is also statistically signifcant. So, it's important to take runs scored and runs allowed when thinking about a teams winning percentage¶
In [109]:
sns.regplot(x='rundiffer', y='WinPCT', data=Teams20)
Out[109]:
performing outlier detection using dffits¶
In [230]:
influence = model1.get_influence()
# Find outliers using DFFITS
number_of_observations = len(Teams20)
number_of_parameters = 2 # parameters include: intercept, x
dffits = influence.dffits[0]
# Use an empirical threshold
dffits_threshold = 2 * np.sqrt(number_of_parameters / number_of_observations)
Teams20[np.abs(dffits) > dffits_threshold]
Out[230]:
found a handful of outliers lets look at how they appear in terms of threshold?¶
In [233]:
ax = sns.regplot(np.arange(len(dffits)),dffits,ci=0,fit_reg=False)
plt.hlines(dffits_threshold,0,30)
plt.hlines(-dffits_threshold,0,30)
Out[233]:
removing outliers¶
In [235]:
cleaned_teams20 = Teams20[np.abs(dffits) < dffits_threshold]
model2 = smf.ols('WinPCT ~ rundiffer', data=cleaned_teams20).fit()
model2.summary()
Out[235]:
after outlier analysis is run and removing them the model gets slightly better we go from and adjusted r square of .84 to .86. Not a very big difference!!¶
In [248]:
sns.regplot(x='rundiffer', y='WinPCT', data=cleaned_teams20)
sns.plt.title('Regression: WinPCT predicted by Run Differential for 2000-2016 (outliers excluded)')
Out[248]:
trying this again but with data from 1871 and so on. This is a very very large amount of observations¶
In [237]:
model3 = smf.ols('WinPCT ~ rundiffer', data=Teams).fit()
model3.summary()
Out[237]:
In [238]:
sns.regplot(x='rundiffer', y='WinPCT', data=Teams)
Out[238]:
In [239]:
influence = model3.get_influence()
# Find outliers using DFFITS
number_of_observations = len(Teams)
number_of_parameters = 2 # parameters include: intercept, x
dffits = influence.dffits[0]
# Use an empirical threshold
dffits_threshold = 2 * np.sqrt(number_of_parameters / number_of_observations)
Teams[np.abs(dffits) > dffits_threshold]
Out[239]:
In [240]:
ax = sns.regplot(np.arange(len(dffits)),dffits,ci=0,fit_reg=False)
plt.hlines(dffits_threshold,0,30)
plt.hlines(-dffits_threshold,0,30)
Out[240]:
so much data but we can see there a handful of outliers that are past the two dark line on our plot¶
here taking out the outliers did make a significant difference our model went from .64 to .848 so appears that the outliers in this observation hold more influence then the previous model. This model also contains more points¶
In [241]:
cleaned_teams = Teams[np.abs(dffits) < dffits_threshold]
model4 = smf.ols('WinPCT ~ rundiffer', data=cleaned_teams).fit()
model4.summary()
Out[241]:
In [247]:
sns.regplot(x='rundiffer', y='WinPCT', data=cleaned_teams)
sns.plt.title('Regression: WinPCT predicted by Run Differential for 1871-2016 (outliers excluded)')
Out[247]:
Question 11: Looking at salaries and teams? Do team with higher Payrolls win more World Series?¶
In [ ]:
# group salary by year id and team
In [215]:
SalbyYrTeam = Salaries[Salaries['yearID'] >= 1985]['salary'].groupby([Salaries['yearID'],Salaries['teamID']]).sum()
In [216]:
# now look at the world series winners
In [217]:
WSW = Teams[(Teams['yearID']>= 1985) & (Teams['WSWin'] == 'Y')][['yearID','teamID','name']]
In [218]:
## itterate through the rows of world series winners, create payroll and pay roll rank
# sal by yr and team has the annual salary : payroll for that year for that team
# for pay roll rank, make sure it's given to the right team id
# (go back and look @ sals some error??)
In [219]:
for index, row in WSW.iterrows():
AnnSals = SalbyYrTeam[row['yearID']]
WSW.set_value(index, 'payroll', AnnSals[row['teamID']])
WSW.set_value(index, 'payroll_rank', AnnSals.rank(ascending=False)[row['teamID']])
go back and check that salaries is corrected. you want to make sure you change CHC to CHN, dang cubbies! They changed their teamid in 2016 to CHC.¶
In [220]:
WSW
Out[220]:
In [221]:
Rank_Wins = WSW['payroll_rank'].value_counts().reset_index()
Rank_Wins.columns = ['payroll_rank', 'winsby_rank']
Rank_Wins
Out[221]:
Looks like times with higher payroll rank in a season win more world series games!¶
In [224]:
p = Bar(Rank_Wins, 'payroll_rank', values='winsby_rank', title="World Series Win by Pay Roll Rank",legend=False)
show(p)
In [231]:
ct = pd.crosstab(Rank_Wins['winsby_rank'],Rank_Wins['payroll_rank'])
ct.head()
Out[231]:
In [232]:
from scipy.stats import chi2_contingency
chi2,p,dof,ex = chi2_contingency(ct)
print('chi2 =', chi2)
print('p-val =', p)
print('degrees of freedom = ', dof)
print('Expected: ')
pd.DataFrame(ex)
Out[232]: